package xin.ganbo.crawler.processor;

import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * 爬取csdn爬虫爬取配置处理器
 * @author ganbo E-mail:bo.gan@56qq.com
 * @since 2017-11-30 14:58
 */

public class CsdnPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    @Override
    public void process(Page page) {
        //判断链接是否符合 http://blog.csdn.net/用户名/article/list/任意个数字
        if (!page.getUrl().regex("http://blog.csdn.net/w+/article/list/[0-9]+").match()) { //添加下面分页信息的列表页url
            String pageLists = page.getHtml().xpath("//*[@id=\"papelist\"]/span").get();    //获取分页信息
            if (StringUtils.isNotBlank(pageLists)) {  //文章数量比较多,底部有分页
                int startPos = pageLists.indexOf("共");
                int endPos = pageLists.indexOf("页");

                //获得CSDN博客的总页数
                int pages = Integer.parseInt(pageLists.substring(startPos + 1, endPos));

                //加入满足条件的链接
                for (int i = 1; i <= pages; i++) {
                    String url = page.getUrl().toString();
                    int index = url.indexOf("list");
                    if(index==-1){
                        page.addTargetRequest(page.getUrl().toString()+"/article/list/" + i);
                    }else{
                        page.addTargetRequest(url.substring(0,url.indexOf("list"))+"list/"+i);
                    }
                }
            } else {  //当文章数量比较小时,就只获取当前页(第一页)
                page.addTargetRequest(page.getUrl().toString().substring(0,page.getUrl().toString().indexOf("list"))+"list/"+1);
            }
        } else {    //获取当前页文章信息
            List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"article_list\"]/div[@class=\"list_item article_item\"]").nodes();
            for (Selectable s : nodes) {
                //获取页面需要的内容
                String title = s.xpath("//*[@class=\"article_title\"]/h1/span/a/text()").toString();
                String url = "http://blog.csdn.net" + s.xpath("//*[@class=\"article_title\"]/h1/span/a/@href").toString();
                String createTime = s.xpath("//*[@class=\"article_manage\"]/span[@class=\"link_postdate\"]/text()").toString();
                String view = s.xpath("//*[@id=\"article_list\"]/div[2]/div[3]/span[2]/text()").toString();
                String comments = s.xpath("//*[@id=\"article_list\"]/div[2]/div[3]/span[3]/text()").toString();
                System.out.println(url + "   " + title + "    " + createTime);
                System.out.println("浏览量===>" + view + "  评论数量===>" + comments);

            }

        }
    }

    @Override
    public Site getSite() {
        return site;
    }

}
